package com.chance.cc.crawler.development.bootstrap.tianmao;

import com.alibaba.fastjson.JSON;
import com.chance.cc.crawler.core.downloader.HttpConfig;
import com.chance.cc.crawler.core.downloader.HttpPage;
import com.chance.cc.crawler.core.downloader.HttpRequest;
import com.chance.cc.crawler.core.downloader.http.HttpClientDownloader;
import com.chance.cc.crawler.core.downloader.http.HttpClientGenerator;
import com.chance.cc.crawler.core.downloader.proxy.SiteConfigProxyProvider;
import com.chance.cc.crawler.core.pipeline.result.CrawlerData;
import com.chance.cc.crawler.core.record.CrawlerResultRecord;
import com.chance.cc.crawler.core.selector.Json;
import com.chance.cc.crawler.development.utils.RedisReader;
import com.jayway.jsonpath.Configuration;
import com.jayway.jsonpath.Option;
import org.apache.commons.io.IOUtils;
import org.apache.commons.lang3.StringUtils;

import java.io.BufferedReader;
import java.io.FileOutputStream;
import java.io.FileReader;
import java.io.IOException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;

/**
 * @author bx
 * @date 2020/12/24 0024 18:23
 */
public class TianmaoSearchProductInfoImport {

    public static final String listQueue = "tianmao_search_result";


    public static void main(String[] args) throws IOException {
//        convertToRedis2();
        syncToMysql();
    }

    private static void convertToRedis() throws IOException {
        FileReader fileReader = new FileReader("D:\\chance\\采集\\电商列表页\\列表\\20210129_tm.json");
        BufferedReader br = new BufferedReader(fileReader);
        List<String> records = new ArrayList<>();
        String line;
        int count = 0;
        RedisReader redisReader = new RedisReader("192.168.1.215", 6379, 4);
        while ((line = br.readLine()) != null){
            count++;
            Json json = new Json(line);
            Configuration conf = Configuration.defaultConfiguration().addOptions(Option.DEFAULT_PATH_LEAF_TO_NULL);
            json.setConfiguration(conf);
            String kw = json.jsonPath("$.关键词").get();
            String itemId = json.jsonPath("$.itemId").get();
            String userId = json.jsonPath("$.sellerId").get();
            Map<String,String> contentMap = new HashMap<>();
            contentMap.put("detailUrl",json.jsonPath("$.URL").get());
            contentMap.put("itemId",json.jsonPath("$.itemId").get());
            contentMap.put("userId",json.jsonPath("$.sellerId").get());
            contentMap.put("itemName",json.jsonPath("$.商品名").get());
            contentMap.put("price",json.jsonPath("$.价格").get());
            contentMap.put("sales",json.jsonPath("$.月销量").get());
            contentMap.put("itemImage",json.jsonPath("$.商品图片").get());
            contentMap.put("shopName",json.jsonPath("$.shop_name").get());
            contentMap.put("keyword",kw);
            contentMap.put("dataId", StringUtils.joinWith("-","tianmao",itemId,userId));
            String contentJson = JSON.toJSONString(contentMap);

            CrawlerResultRecord crawlerResultRecord = new CrawlerResultRecord();
            CrawlerData crawlerData = new CrawlerData();
            crawlerData.setCrawlerContent(contentJson);
            crawlerResultRecord.tagsCreator().bizTags().addCustomKV("keyword",kw);
            crawlerResultRecord.setJsonStr(JSON.toJSONString(crawlerData));


            records.add(JSON.toJSONString(crawlerResultRecord));
            if (records.size() > 1000){
                redisReader.listPush(listQueue, records.toArray(new String[records.size()]));
                records.clear();
            }
        }

        redisReader.listPush(listQueue,records.toArray(new String[records.size()]));

        System.out.println("总条数：" + count);
    }

    private static void convertToRedis2() throws IOException {
        FileReader fileReader = new FileReader("D:\\chance\\采集\\电商列表页\\列表\\20210421.txt");
        BufferedReader br = new BufferedReader(fileReader);
        List<String> records = new ArrayList<>();
        String line;
        int count = 0;
        RedisReader redisReader = new RedisReader("192.168.1.215", 6379, 4);
        while ((line = br.readLine()) != null){
            count++;
            String[] split = line.split("\t");
            String kw = "";
            String itemId = split[0];
            String userId = split[7];
            Map<String,String> contentMap = new HashMap<>();
            contentMap.put("detailUrl",split[5]);
            contentMap.put("itemId",itemId);
            contentMap.put("userId",userId);
            contentMap.put("itemName",split[1]);
            contentMap.put("price",split[3]);
            contentMap.put("sales",split[8]);
            contentMap.put("itemImage",split[9]);
            contentMap.put("shopName",split[4]);
            contentMap.put("keyword",kw);
            contentMap.put("dataId", StringUtils.joinWith("-","tianmao",itemId,userId));
            String contentJson = JSON.toJSONString(contentMap);

            CrawlerResultRecord crawlerResultRecord = new CrawlerResultRecord();
            CrawlerData crawlerData = new CrawlerData();
            crawlerData.setCrawlerContent(contentJson);
            crawlerResultRecord.tagsCreator().bizTags().addCustomKV("keyword",kw);
            crawlerResultRecord.setJsonStr(JSON.toJSONString(crawlerData));


            records.add(JSON.toJSONString(crawlerResultRecord));
            System.out.println(JSON.toJSONString(crawlerResultRecord));
            if (records.size() > 1000){
                redisReader.listPush(listQueue, records.toArray(new String[records.size()]));
                records.clear();
            }
        }

        redisReader.listPush(listQueue,records.toArray(new String[records.size()]));

        System.out.println("总条数：" + count);
    }

    private static void syncToMysql() throws IOException {
        HttpClientDownloader downloader = new HttpClientDownloader();
        downloader.setClientGenerator(new HttpClientGenerator());
        downloader.setProxyProvider(new SiteConfigProxyProvider());

        HttpRequest httpRequest = new HttpRequest();
        httpRequest.setUrl("http://192.168.1.217:9599/crawler/domain/common/api/v1/tianmao/search/results/sync?" +
                "resultQueue="+listQueue);
        HttpPage taobao = downloader.download(httpRequest, HttpConfig.me("tianmao"));
        System.out.println(taobao.getRawText());
    }

    private static void convertToResultJson() throws IOException {
        FileReader fileReader = new FileReader("D:\\chance\\采集\\补采\\tianmao\\tm_shop_top_url.txt");
        BufferedReader br = new BufferedReader(fileReader);
        List<String> lines = new ArrayList<>();
        String line;
        int count = 0;
        while ((line = br.readLine()) != null){
            count++;
            String[] items = line.split("\t");
            Map<String,String> contentMap = new HashMap<>();

            contentMap.put("URL",items[5]);
            contentMap.put("itemId",items[7]);
            contentMap.put("sellerId",items[8]);
            contentMap.put("商品名",items[4]);
            contentMap.put("商品图片",items[10]);
            contentMap.put("shop_name",items[2]);
            contentMap.put("品牌",items[1]);

            String contentJson = JSON.toJSONString(contentMap);
            lines.add(contentJson);
        }

        IOUtils.writeLines(lines,"\r\n",new FileOutputStream("D:\\chance\\采集\\电商列表页\\列表\\20210129_tm.json"),"utf-8");

        System.out.println("总条数：" + count);
    }
}
